package com.tradesolver;

import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.text.DecimalFormat;
import java.text.NumberFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.HashMap;
import java.util.Map;
import java.util.TreeMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.logging.Logger;

import javax.cache.Cache;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Attribute;
import org.jsoup.nodes.Attributes;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.TextNode;
import org.jsoup.select.Elements;

import com.google.appengine.api.taskqueue.Queue;
import com.google.appengine.api.taskqueue.QueueFactory;
import com.google.appengine.api.urlfetch.FetchOptions;
import com.google.appengine.api.urlfetch.HTTPMethod;
import com.google.appengine.api.urlfetch.HTTPRequest;
import com.google.appengine.api.urlfetch.HTTPResponse;
import com.google.appengine.api.urlfetch.URLFetchService;
import com.google.appengine.api.urlfetch.URLFetchServiceFactory;
import com.google.appengine.repackaged.com.google.common.base.Objects;

@SuppressWarnings("serial")
public class PullServlet extends HttpServlet {
  Logger logger = Logger.getLogger(PullServlet.class.getName());

  Dao dao = new Dao();
  CsvService csvService = new CsvService();
  SimpleDateFormat yyyyMMdd = new SimpleDateFormat("yyyyMMdd");

  Cache cache;

  NumberFormat pf = new DecimalFormat("###,###.###");
  NumberFormat cf = new DecimalFormat("##.####");
  NumberFormat vf = new DecimalFormat("###,###,###,###.###");

  String prefix = "http://investing.businessweek.com/businessweek/research/stocks/financials/financials.asp?ticker=";
  String s1 = ":PM&dataset=";
  String s2 = "&period=";
  String s3 = "&currency=native";

  String[][] patterns = { { "incomeStatement", "A" }, { "balanceSheet", "A" }, { "cashFlow", "A" },
      { "incomeStatement", "Q" }, { "balanceSheet", "Q" }, { "cashFlow", "Q" } };

  URLFetchService service = URLFetchServiceFactory.getURLFetchService();

  private static class TickerDataset {
    String ticker;
    String dataset;
    String period;
    Future<HTTPResponse> future;

    public TickerDataset(String ticker, String dataset, String period, Future<HTTPResponse> future) {
      super();
      this.ticker = ticker;
      this.dataset = dataset;
      this.period = period;
      this.future = future;
    }

    @Override
    public String toString() {
      return Objects.toStringHelper(this).add("ticker", ticker).add("dataset", dataset).add("period", period)
          .toString();
    }

  }

  @Override
  public void doPost(HttpServletRequest req, HttpServletResponse resp) throws IOException {

    String action = req.getParameter("action");
    String ticker = req.getParameter("t");
    if (action != null && ticker != null) {
      if ("pullFinancials".equals(action)) {
        pullFinancials(ticker);
      } else if ("pullCorporate".equals(action)) {
        pullCorporate(ticker);
      } else if ("pullTransactions".equals(action)) {
        resp.setContentType("text/html");
        resp.getWriter().println(pullTransactions(ticker));
      }
    } else {
      logger.warning("Doing nothing for action=" + action + " and ticker=" + ticker);
    }
    //resp.setStatus(HttpServletResponse.SC_NO_CONTENT);
  }

  private void removeAttributes(Element e) {
    Attributes attrs = e.attributes();
    for (Attribute attr: attrs) {
      e.removeAttr(attr.getKey());
    }
  }
  
  private String pullTransactions(String ticker) throws MalformedURLException {
    String url = "http://203.177.35.238/Final2/quotes/timesalesout.asp";
    String parms = "estock=" + ticker + "&efrom=0931&eto=0933&ebuyer=&eseller=&especific=&bsubmit=Search";
    HTTPRequest hr = new HTTPRequest(new URL(url), HTTPMethod.POST, FetchOptions.Builder.withDeadline(20));
    hr.setPayload(parms.getBytes());
    Future<HTTPResponse> future = service.fetchAsync(hr);

    String result = null;
    try {
      HTTPResponse res = future.get();
      byte[] content = res.getContent();
      result = new String(content);

      //logger.fine("Parsing result:" + result);

      
      Document doc = Jsoup.parse(result);
      Element table = doc.select("table").first();
      removeAttributes(table);

      Elements trs = table.select("tr");
      for (Element tr: trs) {
        removeAttributes(tr);
      }

      Elements tds = table.select("td");
      for (Element td: tds) {
        removeAttributes(td);
      }

      Elements fonts = table.select("font");
      for (Element font: fonts) {
        if (font.childNodes().isEmpty()) {
          font.remove();
        } else {
          font.parent().appendChild(font.childNode(0));
          font.remove();
        }
      }

      result = table.outerHtml();
      logger.info(result);
      //table = table.nextElementSibling();
//
//      Elements tds = table.select("td");
//      for (Element td : tds) {
//        td.removeAttr("class");
//        td.removeAttr("width");
//        td.removeAttr("align");
//      }
//
//      String html = table.outerHtml();
//      logger.info(html);
//
//      FinancialsText ft = dao.getFinancialsText(ticker);
//      ft.corporate = html;
//      dao.putFinancialsText(ft);

    } catch (InterruptedException e) {
      e.printStackTrace();
      logger.severe(e.getMessage() + " for " + ticker);
    } catch (ExecutionException e) {
      e.printStackTrace();
      logger.severe(e.getMessage() + " for " + ticker);
    }
    return result;
    
  }

  private void pullCorporate(String ticker) throws MalformedURLException {

    String urlPrefix = "http://www.pse.com.ph/html/MarketInformation/stockinfo.jsp?securitySymbol=";

    String url = urlPrefix + ticker;
    logger.info("Fetching url=" + url);
    HTTPRequest hr = new HTTPRequest(new URL(url), HTTPMethod.GET, FetchOptions.Builder.withDeadline(60));
    Future<HTTPResponse> future = service.fetchAsync(hr);

    String result = null;
    try {
      HTTPResponse res = future.get();
      byte[] content = res.getContent();
      result = new String(content);

      logger.fine("Parsing result");

      Document doc = Jsoup.parse(result);
      Element table = doc.select("table.bgcolor_2").first();
      table = table.nextElementSibling();

      Elements tds = table.select("td");
      for (Element td : tds) {
        td.removeAttr("class");
        td.removeAttr("width");
        td.removeAttr("align");
      }

      String html = table.outerHtml();
      logger.info(html);

      FinancialsText ft = dao.getFinancialsText(ticker);
      ft.corporate = html;
      dao.putFinancialsText(ft);

    } catch (InterruptedException e) {
      e.printStackTrace();
      logger.severe(e.getMessage() + " for " + ticker);
    } catch (ExecutionException e) {
      e.printStackTrace();
      logger.severe(e.getMessage() + " for " + ticker);
    }
  }

  private void pullFinancials(String ticker) throws MalformedURLException {
    Map<String, TickerDataset> tickerDatasets = new HashMap<String, TickerDataset>();
    for (String[] pattern : patterns) {
      String url = prefix + ticker + s1 + pattern[0] + s2 + pattern[1] + s3;
      logger.info("Fetching url=" + url);
      HTTPRequest hr = new HTTPRequest(new URL(url), HTTPMethod.GET, FetchOptions.Builder.withDeadline(60));
      Future<HTTPResponse> future = service.fetchAsync(hr);
      tickerDatasets.put(pattern[0] + pattern[1], new TickerDataset(ticker, pattern[0], pattern[1], future));
    }

    FinancialsText ft = new FinancialsText(ticker);
    ft.incomeStatementA = getResponse(tickerDatasets.get("incomeStatementA"));
    ft.balanceSheetA = getResponse(tickerDatasets.get("balanceSheetA"));
    ft.cashFlowA = getResponse(tickerDatasets.get("cashFlowA"));
    ft.incomeStatementQ = getResponse(tickerDatasets.get("incomeStatementQ"));
    ft.balanceSheetQ = getResponse(tickerDatasets.get("balanceSheetQ"));
    ft.cashFlowQ = getResponse(tickerDatasets.get("cashFlowQ"));

    dao.putFinancialsText(ft);
  }

  @Override
  public void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException {

    String action = req.getParameter("action");
    if (action != null) {
      resp.setContentType("text/plain");
      if ("transform".equals(action)) {
        transform(resp);
      } else if ("updateStats".equals(action)) {
        updateStats(resp);
      } else if ("updateCorporateData".equals(action)) {
        updateCorporateData(resp);
      } else if ("pullCorporateAll".equals(action)) {
        Queue queue = QueueFactory.getDefaultQueue();
        for (int i = 0; i < Companies.tickers.length; i++) {
          //if (dao.getFinancialsText(Companies.tickers[i]) == null) {
            queue.add(withUrl("/pull").param("action", "pullCorporate").param("t", Companies.tickers[i]));
          //}
        }
      } else if ("pullFinancialsAll".equals(action)) {
        Queue queue = QueueFactory.getDefaultQueue();
        for (int i = 0; i < Companies.tickers.length; i++) {
          if (dao.getFinancialsText(Companies.tickers[i]) == null) {
            queue.add(withUrl("/pull").param("action", "pullFinancials").param("t", Companies.tickers[i]));
          }
        }
      } else {
        doPost(req, resp);
      }
      return;
    }

    resp.setStatus(HttpServletResponse.SC_NO_CONTENT);
  }

  private TreeMap<String, TreeMap<String, String>> retrieveInfo(byte[] mapBytes) {
    TreeMap<String, TreeMap<String, String>> mapInfo;
    try {
      ByteArrayInputStream bis = new ByteArrayInputStream(mapBytes);
      ObjectInputStream in;
      in = new ObjectInputStream(bis);
      mapInfo = (TreeMap<String, TreeMap<String, String>>) in.readObject();
      in.close();
    } catch (IOException e) {
      throw new IllegalStateException(e);
    } catch (ClassNotFoundException e) {
      throw new IllegalStateException(e);
    }
    return mapInfo;
  }

  private void updateCorporateData(HttpServletResponse resp) throws IOException {
    for (String ticker : Companies.tickers) {
    //String ticker = "SMC";
      logger.info("Processing " + ticker);
      FinancialsText ft = dao.getFinancialsText(ticker);

      FinancialsMap fm = dao.getFinancialsMap(ticker);
      if (ft != null && fm != null) {
        TreeMap<String, TreeMap<String, String>> mapInfo = retrieveInfo(fm.data);

        String lastKey = mapInfo.lastKey();
        if (lastKey != null) {
          TreeMap<String, String> props = mapInfo.get(lastKey);
          props.putAll(parseCorporate(ft.corporate));
          mapInfo.put(lastKey, props);
          
          fm.data = toByteArray(mapInfo);
          dao.putFinancialsMap(fm);
        } else {
          logger.severe("Empty map for FinancialsMap record:" + ticker);
        }

      } else {
        logger.warning("Doing nothing");
      }
    }

  }

  private TreeMap<String, String> parseCorporate(String tableHtml) {
    TreeMap<String, String> result = new TreeMap<String, String>();

    Document doc = Jsoup.parse(tableHtml);
    Element table = doc.select("table").first();

    if (table != null) {
      Elements trs = table.children();
      for (int i = 0; i < 12; i++) {
        Elements tds = trs.get(i).children();
        String key = tds.get(1).text();
        String value = tds.get(2).text();
        result.put(key, value);
        key = tds.get(4).text();
        value = tds.get(5).text();
        result.put(key, value);
      }

    } else {
      logger.fine("no corporate table found");
    }

    return result;
  }

  private void updateStats(HttpServletResponse resp) {
    for (String ticker : Companies.tickers) {
      // String ticker = "SMC";
      logger.info("Processing " + ticker);

      TreeMap<String, TreeMap<String, Quote>> quotesMap = csvService.getQuotes(new String[] { ticker });
      TreeMap<String, Quote> quotes = quotesMap.get(ticker);

      if (quotes != null && !quotes.isEmpty()) {
        String lastKey = quotes.lastKey();
        Quote lastQuote = quotes.get(lastKey);
        if (quotes.lowerKey(lastKey) != null) {
          Quote prevQuote = quotes.get(quotes.lowerKey(lastKey));

          TickerStat tickerStat = dao.getTickerStat(ticker);
          if (tickerStat == null) {
            tickerStat = new TickerStat(ticker);
          }

          try {
            tickerStat.lastDate = yyyyMMdd.parse(lastQuote.quoteDate);
            tickerStat.lastOpen = lastQuote.open;
            tickerStat.lastHigh = lastQuote.high;
            tickerStat.lastLow = lastQuote.low;
            tickerStat.lastClose = lastQuote.close;
            tickerStat.lastVolume = lastQuote.volume;

            tickerStat.prevDate = yyyyMMdd.parse(prevQuote.quoteDate);
            tickerStat.prevOpen = prevQuote.open;
            tickerStat.prevHigh = prevQuote.high;
            tickerStat.prevLow = prevQuote.low;
            tickerStat.prevClose = prevQuote.close;
            tickerStat.prevVolume = prevQuote.volume;

          } catch (ParseException e) {
            throw new RuntimeException(e);
          }

          dao.putTickerStat(tickerStat);
        } else {
          logger.warning("No previous quote data found for " + ticker);
        }
      } else {
        logger.warning("No quotes data found for " + ticker);
      }
    }
  }

  private void transform(HttpServletResponse resp) throws IOException {

    // String ticker = "SMC";

    for (String ticker : Companies.tickers) {
      logger.info("Processing " + ticker);
      FinancialsText ft = dao.getFinancialsText(ticker);

      TreeMap<String, TreeMap<String, String>> incomeStatementA = parseTable(ft.incomeStatementA);

      if (!incomeStatementA.isEmpty()) {
        TreeMap<String, TreeMap<String, String>> balanceSheetA = parseTable(ft.balanceSheetA);

        TreeMap<String, TreeMap<String, String>> merged = new TreeMap<String, TreeMap<String, String>>();

        merged.putAll(incomeStatementA);
        for (String year : balanceSheetA.keySet()) {
          if (merged.containsKey(year)) {
            merged.get(year).putAll(balanceSheetA.get(year));
          } else {
            throw new RuntimeException("Missing key:" + year);
          }
        }

        FinancialsMap fm = new FinancialsMap(ticker);
        fm.year = merged.lastKey();

        fm.data = toByteArray(merged);

        dao.putFinancialsMap(fm);
        logger.info("Done saving");
      } else {
        logger.info("No data for " + ticker);
      }
    }
  }

  private byte[] toByteArray(TreeMap<String, TreeMap<String, String>> merged) throws IOException {
    logger.info("Serializing map");
    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    ObjectOutputStream out = new ObjectOutputStream(bos);
    out.writeObject(merged);
    out.flush();
    out.close();
    logger.info("Done Serializing map");
    return bos.toByteArray();
  }

  private TreeMap<String, TreeMap<String, String>> parseTable(String tableHtml) {
    TreeMap<String, TreeMap<String, String>> result = new TreeMap<String, TreeMap<String, String>>();

    Document doc = Jsoup.parse(tableHtml);
    Element table = doc.select("table.financialStatement").first();

    String[] years = new String[4];
    TreeMap<String, String>[] infos = new TreeMap[4];
    if (table != null) {
      Elements tds = table.child(0).children();
      years[0] = ((TextNode) tds.get(2).childNodes().get(2)).text();
      years[1] = ((TextNode) tds.get(3).childNodes().get(2)).text();
      years[2] = ((TextNode) tds.get(4).childNodes().get(2)).text();
      years[3] = ((TextNode) tds.get(5).childNodes().get(2)).text();

      infos[0] = new TreeMap<String, String>();
      infos[1] = new TreeMap<String, String>();
      infos[2] = new TreeMap<String, String>();
      infos[3] = new TreeMap<String, String>();

      Elements trs = table.children();
      for (int i = 1; i < trs.size(); i++) {
        String key = trs.get(i).child(0).text();
        infos[0].put(key, trs.get(i).child(1).text());
        infos[1].put(key, trs.get(i).child(2).text());
        infos[2].put(key, trs.get(i).child(3).text());
        infos[3].put(key, trs.get(i).child(4).text());
      }

      result.put(years[0], infos[0]);
      result.put(years[1], infos[1]);
      result.put(years[2], infos[2]);
      result.put(years[3], infos[3]);
    } else {
      logger.fine("no financialStatement table found");
    }

    return result;
  }

  private String getResponse(TickerDataset tickerDataset) {
    String result = null;
    try {
      HTTPResponse res = tickerDataset.future.get();
      byte[] content = res.getContent();
      result = new String(content);

      logger.fine("Parsing result");

      Document doc = Jsoup.parse(result);
      Element table = doc.select("table").first();

      Elements tds = table.select("td");
      for (Element td : tds) {
        td.removeAttr("class");
        td.removeAttr("style");
      }

      Elements imgs = table.select("img");
      for (Element img : imgs) {
        img.remove();
      }

      result = table.outerHtml();
    } catch (InterruptedException e) {
      logger.severe(e.getMessage() + " for " + tickerDataset);
    } catch (ExecutionException e) {
      logger.severe(e.getMessage() + " for " + tickerDataset);
    }
    return result;
  }

}
